{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import h2o\n",
    "import time\n",
    "from h2o.estimators.glm import H2OGeneralizedLinearEstimator\n",
    "from h2o.estimators.gbm import H2OGradientBoostingEstimator\n",
    "from h2o.estimators.random_forest import H2ORandomForestEstimator\n",
    "from h2o.estimators.deeplearning import H2ODeepLearningEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime: </td>\n",
       "<td>1 minutes 6 seconds 272 milliseconds </td></tr>\n",
       "<tr><td>H2O cluster version: </td>\n",
       "<td>3.7.0.99999</td></tr>\n",
       "<tr><td>H2O cluster name: </td>\n",
       "<td>ece</td></tr>\n",
       "<tr><td>H2O cluster total nodes: </td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster total memory: </td>\n",
       "<td>8.89 GB</td></tr>\n",
       "<tr><td>H2O cluster total cores: </td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster allowed cores: </td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster healthy: </td>\n",
       "<td>True</td></tr>\n",
       "<tr><td>H2O Connection ip: </td>\n",
       "<td>127.0.0.1</td></tr>\n",
       "<tr><td>H2O Connection port: </td>\n",
       "<td>54321</td></tr>\n",
       "<tr><td>H2O Connection proxy: </td>\n",
       "<td>None</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ------------------------------------\n",
       "H2O cluster uptime:         1 minutes 6 seconds 272 milliseconds\n",
       "H2O cluster version:        3.7.0.99999\n",
       "H2O cluster name:           ece\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster total memory:   8.89 GB\n",
       "H2O cluster total cores:    8\n",
       "H2O cluster allowed cores:  8\n",
       "H2O cluster healthy:        True\n",
       "H2O Connection ip:          127.0.0.1\n",
       "H2O Connection port:        54321\n",
       "H2O Connection proxy:\n",
       "--------------------------  ------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Explore a typical Data Science workflow with H2O and Python\n",
    "#\n",
    "# Goal: assist the manager of CitiBike of NYC to load-balance the bicycles\n",
    "# across the CitiBike network of stations, by predicting the number of bike\n",
    "# trips taken from the station every day.  Use 10 million rows of historical\n",
    "# data, and eventually add weather data.\n",
    "\n",
    "\n",
    "# Connect to a cluster\n",
    "h2o.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from h2o.utils.shared_utils import _locate # private function. used to find files within h2o git project directory.\n",
    "\n",
    "# Set this to True if you want to fetch the data directly from S3.\n",
    "# This is useful if your cluster is running in EC2.\n",
    "data_source_is_s3 = False\n",
    "\n",
    "def mylocate(s):\n",
    "    if data_source_is_s3:\n",
    "        return \"s3n://h2o-public-test-data/\" + s\n",
    "    else:\n",
    "        return _locate(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Import and Parse bike data\n",
      "\n",
      "Parse Progress: [##################################################] 100%\n"
     ]
    }
   ],
   "source": [
    "# Pick either the big or the small demo.\n",
    "# Big data is 10M rows\n",
    "small_test = [mylocate(\"bigdata/laptop/citibike-nyc/2013-10.csv\")]\n",
    "big_test =   [mylocate(\"bigdata/laptop/citibike-nyc/2013-07.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2013-08.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2013-09.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2013-10.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2013-11.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2013-12.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2014-01.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2014-02.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2014-03.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2014-04.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2014-05.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2014-06.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2014-07.csv\"),\n",
    "              mylocate(\"bigdata/laptop/citibike-nyc/2014-08.csv\")]\n",
    "\n",
    "# ----------\n",
    "\n",
    "# 1- Load data - 1 row per bicycle trip.  Has columns showing the start and end\n",
    "# station, trip duration and trip start time and day.  The larger dataset\n",
    "# totals about 10 million rows\n",
    "print(\"Import and Parse bike data\")\n",
    "data = h2o.import_file(path=big_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows:10,407,546 Cols:16\n",
      "\n",
      "Chunk compression summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>chunk_type</b></td>\n",
       "<td><b>chunk_name</b></td>\n",
       "<td><b>count</b></td>\n",
       "<td><b>count_percentage</b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>size_percentage</b></td></tr>\n",
       "<tr><td>C0L</td>\n",
       "<td>Constant Integers</td>\n",
       "<td>117</td>\n",
       "<td>1.5298117</td>\n",
       "<td>    9.1 KB</td>\n",
       "<td>0.0015501</td></tr>\n",
       "<tr><td>C1</td>\n",
       "<td>1-Byte Integers</td>\n",
       "<td>478</td>\n",
       "<td>6.25</td>\n",
       "<td>   10.0 MB</td>\n",
       "<td>1.7289143</td></tr>\n",
       "<tr><td>C1N</td>\n",
       "<td>1-Byte Integers (w/o NAs)</td>\n",
       "<td>478</td>\n",
       "<td>6.25</td>\n",
       "<td>   10.0 MB</td>\n",
       "<td>1.7289143</td></tr>\n",
       "<tr><td>C1S</td>\n",
       "<td>1-Byte Fractions</td>\n",
       "<td>839</td>\n",
       "<td>10.970188</td>\n",
       "<td>   17.5 MB</td>\n",
       "<td>3.042758</td></tr>\n",
       "<tr><td>C2</td>\n",
       "<td>2-Byte Integers</td>\n",
       "<td>2616</td>\n",
       "<td>34.20502</td>\n",
       "<td>  108.8 MB</td>\n",
       "<td>18.8909</td></tr>\n",
       "<tr><td>C2S</td>\n",
       "<td>2-Byte Fractions</td>\n",
       "<td>314</td>\n",
       "<td>4.1056485</td>\n",
       "<td>   12.9 MB</td>\n",
       "<td>2.2460942</td></tr>\n",
       "<tr><td>C4</td>\n",
       "<td>4-Byte Integers</td>\n",
       "<td>214</td>\n",
       "<td>2.7981172</td>\n",
       "<td>   17.9 MB</td>\n",
       "<td>3.1005228</td></tr>\n",
       "<tr><td>C4S</td>\n",
       "<td>4-Byte Fractions</td>\n",
       "<td>389</td>\n",
       "<td>5.086297</td>\n",
       "<td>   32.4 MB</td>\n",
       "<td>5.625424</td></tr>\n",
       "<tr><td>C8</td>\n",
       "<td>64-bit Integers</td>\n",
       "<td>680</td>\n",
       "<td>8.891213</td>\n",
       "<td>  113.5 MB</td>\n",
       "<td>19.704786</td></tr>\n",
       "<tr><td>C8D</td>\n",
       "<td>64-bit Reals</td>\n",
       "<td>1523</td>\n",
       "<td>19.913704</td>\n",
       "<td>  253.0 MB</td>\n",
       "<td>43.930134</td></tr></table></div>"
      ],
      "text/plain": [
       "chunk_type    chunk_name                 count    count_percentage    size      size_percentage\n",
       "------------  -------------------------  -------  ------------------  --------  -----------------\n",
       "C0L           Constant Integers          117      1.52981             9.1 KB    0.00155005\n",
       "C1            1-Byte Integers            478      6.25                10.0 MB   1.72891\n",
       "C1N           1-Byte Integers (w/o NAs)  478      6.25                10.0 MB   1.72891\n",
       "C1S           1-Byte Fractions           839      10.9702             17.5 MB   3.04276\n",
       "C2            2-Byte Integers            2616     34.205              108.8 MB  18.8909\n",
       "C2S           2-Byte Fractions           314      4.10565             12.9 MB   2.24609\n",
       "C4            4-Byte Integers            214      2.79812             17.9 MB   3.10052\n",
       "C4S           4-Byte Fractions           389      5.0863              32.4 MB   5.62542\n",
       "C8            64-bit Integers            680      8.89121             113.5 MB  19.7048\n",
       "C8D           64-bit Reals               1523     19.9137             253.0 MB  43.9301"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Frame distribution summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>number_of_rows</b></td>\n",
       "<td><b>number_of_chunks_per_column</b></td>\n",
       "<td><b>number_of_chunks</b></td></tr>\n",
       "<tr><td>172.16.2.61:54321</td>\n",
       "<td>  575.9 MB</td>\n",
       "<td>10407546.0000000</td>\n",
       "<td>478.0</td>\n",
       "<td>7648.0</td></tr>\n",
       "<tr><td>mean</td>\n",
       "<td>  575.9 MB</td>\n",
       "<td>10407546.0000000</td>\n",
       "<td>478.0</td>\n",
       "<td>7648.0</td></tr>\n",
       "<tr><td>min</td>\n",
       "<td>  575.9 MB</td>\n",
       "<td>10407546.0000000</td>\n",
       "<td>478.0</td>\n",
       "<td>7648.0</td></tr>\n",
       "<tr><td>max</td>\n",
       "<td>  575.9 MB</td>\n",
       "<td>10407546.0000000</td>\n",
       "<td>478.0</td>\n",
       "<td>7648.0</td></tr>\n",
       "<tr><td>stddev</td>\n",
       "<td>      0  B</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>total</td>\n",
       "<td>  575.9 MB</td>\n",
       "<td>10407546.0000000</td>\n",
       "<td>478.0</td>\n",
       "<td>7648.0</td></tr></table></div>"
      ],
      "text/plain": [
       "                   size      number_of_rows    number_of_chunks_per_column    number_of_chunks\n",
       "-----------------  --------  ----------------  -----------------------------  ------------------\n",
       "172.16.2.61:54321  575.9 MB  1.04075e+07       478                            7648\n",
       "mean               575.9 MB  1.04075e+07       478                            7648\n",
       "min                575.9 MB  1.04075e+07       478                            7648\n",
       "max                575.9 MB  1.04075e+07       478                            7648\n",
       "stddev             0  B      0                 0                              0\n",
       "total              575.9 MB  1.04075e+07       478                            7648"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>       </th><th>tripduration  </th><th>starttime        </th><th>stoptime         </th><th>start station id  </th><th>start station name     </th><th>start station latitude  </th><th>start station longitude  </th><th>end station id  </th><th>end station name           </th><th>end station latitude  </th><th>end station longitude  </th><th>bikeid       </th><th>usertype      </th><th>birth year   </th><th>gender        </th><th>Days         </th></tr>\n",
       "<tr><td>type   </td><td>int           </td><td>time             </td><td>time             </td><td>int               </td><td>enum                   </td><td>real                    </td><td>real                     </td><td>int             </td><td>enum                       </td><td>real                  </td><td>real                   </td><td>int          </td><td>enum          </td><td>int          </td><td>int           </td><td>int          </td></tr>\n",
       "<tr><td>mins   </td><td>60.0          </td><td>1.372662e+12     </td><td>1.372662242e+12  </td><td>72.0              </td><td>0.0                    </td><td>40.680342423            </td><td>-74.01713445             </td><td>72.0            </td><td>0.0                        </td><td>40.680342423          </td><td>-74.01713445           </td><td>14529.0      </td><td>0.0           </td><td>1899.0       </td><td>0.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>mean   </td><td>868.968726057 </td><td>1.39099985823e+12</td><td>1.39100072718e+12</td><td>444.858835407     </td><td>NaN                    </td><td>40.7343819823           </td><td>-73.9910570182           </td><td>445.259785544   </td><td>NaN                        </td><td>40.7340868895         </td><td>-73.9911707799         </td><td>17895.6618358</td><td>0.880131781306</td><td>1975.79892394</td><td>1.08414654137 </td><td>16099.0245177</td></tr>\n",
       "<tr><td>maxs   </td><td>6250750.0     </td><td>1.409554787e+12  </td><td>1.409563605e+12  </td><td>3002.0            </td><td>339.0                  </td><td>40.771522               </td><td>-73.9500479759           </td><td>3002.0          </td><td>339.0                      </td><td>40.771522             </td><td>-73.9500479759         </td><td>21689.0      </td><td>1.0           </td><td>1998.0       </td><td>2.0           </td><td>16314.0      </td></tr>\n",
       "<tr><td>sigma  </td><td>2985.10540532 </td><td>11806578171.7    </td><td>11806555707.8    </td><td>355.755989765     </td><td>NaN                    </td><td>0.0197100508736         </td><td>0.0123453320185          </td><td>360.070380844   </td><td>NaN                        </td><td>0.0197309578633       </td><td>0.0124311861598        </td><td>1938.80517884</td><td>0.324807387506</td><td>11.132784905 </td><td>0.563019777794</td><td>136.647269305</td></tr>\n",
       "<tr><td>zeros  </td><td>0             </td><td>0                </td><td>0                </td><td>0                 </td><td>56836                  </td><td>0                       </td><td>0                        </td><td>0               </td><td>55167                      </td><td>0                     </td><td>0                      </td><td>0            </td><td>1247534       </td><td>0            </td><td>1248517       </td><td>0            </td></tr>\n",
       "<tr><td>missing</td><td>0             </td><td>0                </td><td>0                </td><td>0                 </td><td>0                      </td><td>0                       </td><td>0                        </td><td>0               </td><td>0                          </td><td>0                     </td><td>0                      </td><td>0            </td><td>0             </td><td>1247644      </td><td>0             </td><td>0            </td></tr>\n",
       "<tr><td>0      </td><td>634.0         </td><td>1.372662e+12     </td><td>1.372662634e+12  </td><td>164.0             </td><td>E 47 St & 2 Ave        </td><td>40.75323098             </td><td>-73.97032517             </td><td>504.0           </td><td>1 Ave & E 15 St            </td><td>40.73221853           </td><td>-73.98165557           </td><td>16950.0      </td><td>Customer      </td><td>nan          </td><td>0.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>1      </td><td>1547.0        </td><td>1.372662002e+12  </td><td>1.372663549e+12  </td><td>388.0             </td><td>W 26 St & 10 Ave       </td><td>40.749717753            </td><td>-74.002950346            </td><td>459.0           </td><td>W 20 St & 11 Ave           </td><td>40.746745             </td><td>-74.007756             </td><td>19816.0      </td><td>Customer      </td><td>nan          </td><td>0.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>2      </td><td>178.0         </td><td>1.372662064e+12  </td><td>1.372662242e+12  </td><td>293.0             </td><td>Lafayette St & E 8 St  </td><td>40.73028666             </td><td>-73.9907647              </td><td>237.0           </td><td>E 11 St & 2 Ave            </td><td>40.73047309           </td><td>-73.98672378           </td><td>14548.0      </td><td>Subscriber    </td><td>1980.0       </td><td>2.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>3      </td><td>1580.0        </td><td>1.372662066e+12  </td><td>1.372663646e+12  </td><td>531.0             </td><td>Forsyth St & Broome St </td><td>40.71893904             </td><td>-73.99266288             </td><td>499.0           </td><td>Broadway & W 60 St         </td><td>40.76915505           </td><td>-73.98191841           </td><td>16063.0      </td><td>Customer      </td><td>nan          </td><td>0.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>4      </td><td>757.0         </td><td>1.37266207e+12   </td><td>1.372662827e+12  </td><td>382.0             </td><td>University Pl & E 14 St</td><td>40.73492695             </td><td>-73.99200509             </td><td>410.0           </td><td>Suffolk St & Stanton St    </td><td>40.72066442           </td><td>-73.98517977           </td><td>19213.0      </td><td>Subscriber    </td><td>1986.0       </td><td>1.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>5      </td><td>861.0         </td><td>1.372662083e+12  </td><td>1.372662944e+12  </td><td>511.0             </td><td>E 14 St & Avenue B     </td><td>40.72938685             </td><td>-73.97772429             </td><td>454.0           </td><td>E 51 St & 1 Ave            </td><td>40.75455731           </td><td>-73.96592976           </td><td>16223.0      </td><td>Subscriber    </td><td>1988.0       </td><td>1.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>6      </td><td>550.0         </td><td>1.372662119e+12  </td><td>1.372662669e+12  </td><td>293.0             </td><td>Lafayette St & E 8 St  </td><td>40.73028666             </td><td>-73.9907647              </td><td>394.0           </td><td>E 9 St & Avenue C          </td><td>40.72521311           </td><td>-73.97768752           </td><td>16746.0      </td><td>Customer      </td><td>nan          </td><td>0.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>7      </td><td>288.0         </td><td>1.372662136e+12  </td><td>1.372662424e+12  </td><td>224.0             </td><td>Spruce St & Nassau St  </td><td>40.71146364             </td><td>-74.00552427             </td><td>376.0           </td><td>John St & William St       </td><td>40.70862144           </td><td>-74.00722156           </td><td>16062.0      </td><td>Subscriber    </td><td>1985.0       </td><td>2.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>8      </td><td>766.0         </td><td>1.372662136e+12  </td><td>1.372662902e+12  </td><td>432.0             </td><td>E 7 St & Avenue A      </td><td>40.72621788             </td><td>-73.98379855             </td><td>336.0           </td><td>Sullivan St & Washington Sq</td><td>40.73047747           </td><td>-73.99906065           </td><td>17963.0      </td><td>Subscriber    </td><td>1980.0       </td><td>2.0           </td><td>15887.0      </td></tr>\n",
       "<tr><td>9      </td><td>773.0         </td><td>1.372662143e+12  </td><td>1.372662916e+12  </td><td>173.0             </td><td>Broadway & W 49 St     </td><td>40.76064679             </td><td>-73.98442659             </td><td>479.0           </td><td>9 Ave & W 45 St            </td><td>40.76019252           </td><td>-73.9912551            </td><td>19365.0      </td><td>Subscriber    </td><td>1989.0       </td><td>1.0           </td><td>15887.0      </td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ----------\n",
    "\n",
    "# 2- light data munging: group the bike starts per-day, converting the 10M rows\n",
    "# of trips to about 140,000 station&day combos - predicting the number of trip\n",
    "# starts per-station-per-day.\n",
    "\n",
    "# Convert start time to: Day since the Epoch\n",
    "startime = data[\"starttime\"]\n",
    "secsPerDay=1000*60*60*24\n",
    "data[\"Days\"] = (startime/secsPerDay).floor()\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th style=\"text-align: right;\">  Days</th><th>start station name  </th><th style=\"text-align: right;\">  bikes</th></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>1 Ave & E 15 St     </td><td style=\"text-align: right;\">     47</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>1 Ave & E 18 St     </td><td style=\"text-align: right;\">     40</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>1 Ave & E 30 St     </td><td style=\"text-align: right;\">     42</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>1 Ave & E 44 St     </td><td style=\"text-align: right;\">     12</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>10 Ave & W 28 St    </td><td style=\"text-align: right;\">     28</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>11 Ave & W 27 St    </td><td style=\"text-align: right;\">     30</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>11 Ave & W 41 St    </td><td style=\"text-align: right;\">     20</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>12 Ave & W 40 St    </td><td style=\"text-align: right;\">     13</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>2 Ave & E 31 St     </td><td style=\"text-align: right;\">     33</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>2 Ave & E 58 St     </td><td style=\"text-align: right;\">     54</td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows:139,261 Cols:3\n",
      "\n",
      "Chunk compression summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>chunk_type</b></td>\n",
       "<td><b>chunk_name</b></td>\n",
       "<td><b>count</b></td>\n",
       "<td><b>count_percentage</b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>size_percentage</b></td></tr>\n",
       "<tr><td>C1N</td>\n",
       "<td>1-Byte Integers (w/o NAs)</td>\n",
       "<td>2</td>\n",
       "<td>2.0833335</td>\n",
       "<td>    8.6 KB</td>\n",
       "<td>1.2723082</td></tr>\n",
       "<tr><td>C1S</td>\n",
       "<td>1-Byte Fractions</td>\n",
       "<td>32</td>\n",
       "<td>33.333336</td>\n",
       "<td>  138.6 KB</td>\n",
       "<td>20.434813</td></tr>\n",
       "<tr><td>C2</td>\n",
       "<td>2-Byte Integers</td>\n",
       "<td>62</td>\n",
       "<td>64.58333</td>\n",
       "<td>  531.1 KB</td>\n",
       "<td>78.29288</td></tr></table></div>"
      ],
      "text/plain": [
       "chunk_type    chunk_name                 count    count_percentage    size      size_percentage\n",
       "------------  -------------------------  -------  ------------------  --------  -----------------\n",
       "C1N           1-Byte Integers (w/o NAs)  2        2.08333             8.6 KB    1.27231\n",
       "C1S           1-Byte Fractions           32       33.3333             138.6 KB  20.4348\n",
       "C2            2-Byte Integers            62       64.5833             531.1 KB  78.2929"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Frame distribution summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>number_of_rows</b></td>\n",
       "<td><b>number_of_chunks_per_column</b></td>\n",
       "<td><b>number_of_chunks</b></td></tr>\n",
       "<tr><td>172.16.2.61:54321</td>\n",
       "<td>  678.4 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>96.0</td></tr>\n",
       "<tr><td>mean</td>\n",
       "<td>  678.4 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>96.0</td></tr>\n",
       "<tr><td>min</td>\n",
       "<td>  678.4 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>96.0</td></tr>\n",
       "<tr><td>max</td>\n",
       "<td>  678.4 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>96.0</td></tr>\n",
       "<tr><td>stddev</td>\n",
       "<td>      0  B</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>total</td>\n",
       "<td>  678.4 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>96.0</td></tr></table></div>"
      ],
      "text/plain": [
       "                   size      number_of_rows    number_of_chunks_per_column    number_of_chunks\n",
       "-----------------  --------  ----------------  -----------------------------  ------------------\n",
       "172.16.2.61:54321  678.4 KB  139261            32                             96\n",
       "mean               678.4 KB  139261            32                             96\n",
       "min                678.4 KB  139261            32                             96\n",
       "max                678.4 KB  139261            32                             96\n",
       "stddev             0  B      0                 0                              0\n",
       "total              678.4 KB  139261            32                             96"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>       </th><th>Days         </th><th>start station name  </th><th>bikes        </th></tr>\n",
       "<tr><td>type   </td><td>int          </td><td>enum                </td><td>int          </td></tr>\n",
       "<tr><td>mins   </td><td>15887.0      </td><td>0.0                 </td><td>1.0          </td></tr>\n",
       "<tr><td>mean   </td><td>16099.9758008</td><td>NaN                 </td><td>74.7329833909</td></tr>\n",
       "<tr><td>maxs   </td><td>16314.0      </td><td>339.0               </td><td>680.0        </td></tr>\n",
       "<tr><td>sigma  </td><td>123.635133897</td><td>NaN                 </td><td>64.1241591283</td></tr>\n",
       "<tr><td>zeros  </td><td>0            </td><td>428                 </td><td>0            </td></tr>\n",
       "<tr><td>missing</td><td>0            </td><td>0                   </td><td>0            </td></tr>\n",
       "<tr><td>0      </td><td>15887.0      </td><td>1 Ave & E 15 St     </td><td>47.0         </td></tr>\n",
       "<tr><td>1      </td><td>15887.0      </td><td>1 Ave & E 18 St     </td><td>40.0         </td></tr>\n",
       "<tr><td>2      </td><td>15887.0      </td><td>1 Ave & E 30 St     </td><td>42.0         </td></tr>\n",
       "<tr><td>3      </td><td>15887.0      </td><td>1 Ave & E 44 St     </td><td>12.0         </td></tr>\n",
       "<tr><td>4      </td><td>15887.0      </td><td>10 Ave & W 28 St    </td><td>28.0         </td></tr>\n",
       "<tr><td>5      </td><td>15887.0      </td><td>11 Ave & W 27 St    </td><td>30.0         </td></tr>\n",
       "<tr><td>6      </td><td>15887.0      </td><td>11 Ave & W 41 St    </td><td>20.0         </td></tr>\n",
       "<tr><td>7      </td><td>15887.0      </td><td>12 Ave & W 40 St    </td><td>13.0         </td></tr>\n",
       "<tr><td>8      </td><td>15887.0      </td><td>2 Ave & E 31 St     </td><td>33.0         </td></tr>\n",
       "<tr><td>9      </td><td>15887.0      </td><td>2 Ave & E 58 St     </td><td>54.0         </td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[139261, 3]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now do a monster Group-By.  Count bike starts per-station per-day.  Ends up\n",
    "# with about 340 stations times 400 days (140,000 rows).  This is what we want\n",
    "# to predict.\n",
    "grouped = data.group_by([\"Days\",\"start station name\"])\n",
    "bpd = grouped.count().get_frame() # Compute bikes-per-day\n",
    "bpd.set_name(2,\"bikes\")\n",
    "bpd.show()\n",
    "bpd.describe()\n",
    "bpd.dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Quantiles of bikes-per-day\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th style=\"text-align: right;\">  Probs</th><th style=\"text-align: right;\">  bikesQuantiles</th></tr>\n",
       "<tr><td style=\"text-align: right;\">  0.01 </td><td style=\"text-align: right;\">               2</td></tr>\n",
       "<tr><td style=\"text-align: right;\">  0.1  </td><td style=\"text-align: right;\">              11</td></tr>\n",
       "<tr><td style=\"text-align: right;\">  0.25 </td><td style=\"text-align: right;\">              26</td></tr>\n",
       "<tr><td style=\"text-align: right;\">  0.333</td><td style=\"text-align: right;\">              35</td></tr>\n",
       "<tr><td style=\"text-align: right;\">  0.5  </td><td style=\"text-align: right;\">              58</td></tr>\n",
       "<tr><td style=\"text-align: right;\">  0.667</td><td style=\"text-align: right;\">              89</td></tr>\n",
       "<tr><td style=\"text-align: right;\">  0.75 </td><td style=\"text-align: right;\">             107</td></tr>\n",
       "<tr><td style=\"text-align: right;\">  0.9  </td><td style=\"text-align: right;\">             157</td></tr>\n",
       "<tr><td style=\"text-align: right;\">  0.99 </td><td style=\"text-align: right;\">             291</td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Quantiles: the data is fairly unbalanced; some station/day combos are wildly\n",
    "# more popular than others.\n",
    "print(\"Quantiles of bikes-per-day\")\n",
    "bpd[\"bikes\"].quantile().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bikes-Per-Day\n",
      "Rows:139,261 Cols:5\n",
      "\n",
      "Chunk compression summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>chunk_type</b></td>\n",
       "<td><b>chunk_name</b></td>\n",
       "<td><b>count</b></td>\n",
       "<td><b>count_percentage</b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>size_percentage</b></td></tr>\n",
       "<tr><td>C0L</td>\n",
       "<td>Constant Integers</td>\n",
       "<td>18</td>\n",
       "<td>11.25</td>\n",
       "<td>    1.4 KB</td>\n",
       "<td>0.1607895</td></tr>\n",
       "<tr><td>CBS</td>\n",
       "<td>Bits</td>\n",
       "<td>1</td>\n",
       "<td>0.625</td>\n",
       "<td>    614  B</td>\n",
       "<td>0.0685588</td></tr>\n",
       "<tr><td>C1N</td>\n",
       "<td>1-Byte Integers (w/o NAs)</td>\n",
       "<td>47</td>\n",
       "<td>29.374998</td>\n",
       "<td>  202.9 KB</td>\n",
       "<td>23.194105</td></tr>\n",
       "<tr><td>C1S</td>\n",
       "<td>1-Byte Fractions</td>\n",
       "<td>32</td>\n",
       "<td>20.0</td>\n",
       "<td>  138.6 KB</td>\n",
       "<td>15.849935</td></tr>\n",
       "<tr><td>C2</td>\n",
       "<td>2-Byte Integers</td>\n",
       "<td>62</td>\n",
       "<td>38.75</td>\n",
       "<td>  531.1 KB</td>\n",
       "<td>60.726612</td></tr></table></div>"
      ],
      "text/plain": [
       "chunk_type    chunk_name                 count    count_percentage    size      size_percentage\n",
       "------------  -------------------------  -------  ------------------  --------  -----------------\n",
       "C0L           Constant Integers          18       11.25               1.4 KB    0.160789\n",
       "CBS           Bits                       1        0.625               614  B    0.0685588\n",
       "C1N           1-Byte Integers (w/o NAs)  47       29.375              202.9 KB  23.1941\n",
       "C1S           1-Byte Fractions           32       20                  138.6 KB  15.8499\n",
       "C2            2-Byte Integers            62       38.75               531.1 KB  60.7266"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Frame distribution summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>number_of_rows</b></td>\n",
       "<td><b>number_of_chunks_per_column</b></td>\n",
       "<td><b>number_of_chunks</b></td></tr>\n",
       "<tr><td>172.16.2.61:54321</td>\n",
       "<td>  874.6 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>160.0</td></tr>\n",
       "<tr><td>mean</td>\n",
       "<td>  874.6 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>160.0</td></tr>\n",
       "<tr><td>min</td>\n",
       "<td>  874.6 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>160.0</td></tr>\n",
       "<tr><td>max</td>\n",
       "<td>  874.6 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>160.0</td></tr>\n",
       "<tr><td>stddev</td>\n",
       "<td>      0  B</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>total</td>\n",
       "<td>  874.6 KB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>160.0</td></tr></table></div>"
      ],
      "text/plain": [
       "                   size      number_of_rows    number_of_chunks_per_column    number_of_chunks\n",
       "-----------------  --------  ----------------  -----------------------------  ------------------\n",
       "172.16.2.61:54321  874.6 KB  139261            32                             160\n",
       "mean               874.6 KB  139261            32                             160\n",
       "min                874.6 KB  139261            32                             160\n",
       "max                874.6 KB  139261            32                             160\n",
       "stddev             0  B      0                 0                              0\n",
       "total              874.6 KB  139261            32                             160"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>       </th><th>Days         </th><th>start station name  </th><th>bikes        </th><th>Month  </th><th>DayOfWeek  </th></tr>\n",
       "<tr><td>type   </td><td>int          </td><td>enum                </td><td>int          </td><td>enum   </td><td>enum       </td></tr>\n",
       "<tr><td>mins   </td><td>15887.0      </td><td>0.0                 </td><td>1.0          </td><td>0.0    </td><td>0.0        </td></tr>\n",
       "<tr><td>mean   </td><td>16099.9758008</td><td>NaN                 </td><td>74.7329833909</td><td>NaN    </td><td>NaN        </td></tr>\n",
       "<tr><td>maxs   </td><td>16314.0      </td><td>339.0               </td><td>680.0        </td><td>11.0   </td><td>6.0        </td></tr>\n",
       "<tr><td>sigma  </td><td>123.635133897</td><td>NaN                 </td><td>64.1241591283</td><td>NaN    </td><td>NaN        </td></tr>\n",
       "<tr><td>zeros  </td><td>0            </td><td>428                 </td><td>0            </td><td>9949   </td><td>19880      </td></tr>\n",
       "<tr><td>missing</td><td>0            </td><td>0                   </td><td>0            </td><td>0      </td><td>0          </td></tr>\n",
       "<tr><td>0      </td><td>15887.0      </td><td>1 Ave & E 15 St     </td><td>47.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "<tr><td>1      </td><td>15887.0      </td><td>1 Ave & E 18 St     </td><td>40.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "<tr><td>2      </td><td>15887.0      </td><td>1 Ave & E 30 St     </td><td>42.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "<tr><td>3      </td><td>15887.0      </td><td>1 Ave & E 44 St     </td><td>12.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "<tr><td>4      </td><td>15887.0      </td><td>10 Ave & W 28 St    </td><td>28.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "<tr><td>5      </td><td>15887.0      </td><td>11 Ave & W 27 St    </td><td>30.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "<tr><td>6      </td><td>15887.0      </td><td>11 Ave & W 41 St    </td><td>20.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "<tr><td>7      </td><td>15887.0      </td><td>12 Ave & W 40 St    </td><td>13.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "<tr><td>8      </td><td>15887.0      </td><td>2 Ave & E 31 St     </td><td>33.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "<tr><td>9      </td><td>15887.0      </td><td>2 Ave & E 58 St     </td><td>54.0         </td><td>6      </td><td>Sun        </td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# A little feature engineering\n",
    "# Add in month-of-year (seasonality; fewer bike rides in winter than summer)\n",
    "secs = bpd[\"Days\"]*secsPerDay\n",
    "bpd[\"Month\"]     = secs.month().asfactor()\n",
    "# Add in day-of-week (work-week; more bike rides on Sunday than Monday)\n",
    "bpd[\"DayOfWeek\"] = secs.dayOfWeek()\n",
    "print(\"Bikes-Per-Day\")\n",
    "bpd.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ----------\n",
    "# 3- Fit a model on train; using test as validation\n",
    "\n",
    "# Function for doing class test/train/holdout split\n",
    "def split_fit_predict(data):\n",
    "  global gbm0,drf0,glm0,dl0\n",
    "  # Classic Test/Train split\n",
    "  r = data['Days'].runif()   # Random UNIForm numbers, one per row\n",
    "  train = data[  r  < 0.6]\n",
    "  test  = data[(0.6 <= r) & (r < 0.9)]\n",
    "  hold  = data[ 0.9 <= r ]\n",
    "  print(\"Training data has\",train.ncol,\"columns and\",train.nrow,\"rows, test has\",test.nrow,\"rows, holdout has\",hold.nrow)\n",
    "  bike_names_x = data.names\n",
    "  bike_names_x.remove(\"bikes\")\n",
    "  \n",
    "  # Run GBM\n",
    "  s = time.time()\n",
    "  \n",
    "  gbm0 = H2OGradientBoostingEstimator(ntrees=500, # 500 works well\n",
    "                                      max_depth=6,\n",
    "                                      learn_rate=0.1)\n",
    "    \n",
    "\n",
    "  gbm0.train(x               =bike_names_x,\n",
    "             y               =\"bikes\",\n",
    "             training_frame  =train,\n",
    "             validation_frame=test)\n",
    "\n",
    "  gbm_elapsed = time.time() - s\n",
    "\n",
    "  # Run DRF\n",
    "  s = time.time()\n",
    "    \n",
    "  drf0 = H2ORandomForestEstimator(ntrees=250, max_depth=30)\n",
    "\n",
    "  drf0.train(x               =bike_names_x,\n",
    "             y               =\"bikes\",\n",
    "             training_frame  =train,\n",
    "             validation_frame=test)\n",
    "    \n",
    "  drf_elapsed = time.time() - s \n",
    "    \n",
    "    \n",
    "  # Run GLM\n",
    "  if \"WC1\" in bike_names_x: bike_names_x.remove(\"WC1\")\n",
    "  s = time.time()\n",
    "\n",
    "  glm0 = H2OGeneralizedLinearEstimator(Lambda=[1e-5], family=\"poisson\")\n",
    "    \n",
    "  glm0.train(x               =bike_names_x,\n",
    "             y               =\"bikes\",\n",
    "             training_frame  =train,\n",
    "             validation_frame=test)\n",
    "\n",
    "  glm_elapsed = time.time() - s\n",
    "  \n",
    "  # Run DL\n",
    "  s = time.time()\n",
    "\n",
    "  dl0 = H2ODeepLearningEstimator(hidden=[50,50,50,50], epochs=50)\n",
    "    \n",
    "  dl0.train(x               =bike_names_x,\n",
    "            y               =\"bikes\",\n",
    "            training_frame  =train,\n",
    "            validation_frame=test)\n",
    "    \n",
    "  dl_elapsed = time.time() - s\n",
    "  \n",
    "  # ----------\n",
    "  # 4- Score on holdout set & report\n",
    "  train_r2_gbm = gbm0.model_performance(train).r2()\n",
    "  test_r2_gbm  = gbm0.model_performance(test ).r2()\n",
    "  hold_r2_gbm  = gbm0.model_performance(hold ).r2()\n",
    "#   print \"GBM R2 TRAIN=\",train_r2_gbm,\", R2 TEST=\",test_r2_gbm,\", R2 HOLDOUT=\",hold_r2_gbm\n",
    "  \n",
    "  train_r2_drf = drf0.model_performance(train).r2()\n",
    "  test_r2_drf  = drf0.model_performance(test ).r2()\n",
    "  hold_r2_drf  = drf0.model_performance(hold ).r2()\n",
    "#   print \"DRF R2 TRAIN=\",train_r2_drf,\", R2 TEST=\",test_r2_drf,\", R2 HOLDOUT=\",hold_r2_drf\n",
    "  \n",
    "  train_r2_glm = glm0.model_performance(train).r2()\n",
    "  test_r2_glm  = glm0.model_performance(test ).r2()\n",
    "  hold_r2_glm  = glm0.model_performance(hold ).r2()\n",
    "#   print \"GLM R2 TRAIN=\",train_r2_glm,\", R2 TEST=\",test_r2_glm,\", R2 HOLDOUT=\",hold_r2_glm\n",
    "    \n",
    "  train_r2_dl = dl0.model_performance(train).r2()\n",
    "  test_r2_dl  = dl0.model_performance(test ).r2()\n",
    "  hold_r2_dl  = dl0.model_performance(hold ).r2()\n",
    "#   print \" DL R2 TRAIN=\",train_r2_dl,\", R2 TEST=\",test_r2_dl,\", R2 HOLDOUT=\",hold_r2_dl\n",
    "    \n",
    "  # make a pretty HTML table printout of the results\n",
    "\n",
    "  header = [\"Model\", \"R2 TRAIN\", \"R2 TEST\", \"R2 HOLDOUT\", \"Model Training Time (s)\"]\n",
    "  table  = [\n",
    "            [\"GBM\", train_r2_gbm, test_r2_gbm, hold_r2_gbm, round(gbm_elapsed,3)],\n",
    "            [\"DRF\", train_r2_drf, test_r2_drf, hold_r2_drf, round(drf_elapsed,3)],\n",
    "            [\"GLM\", train_r2_glm, test_r2_glm, hold_r2_glm, round(glm_elapsed,3)],\n",
    "            [\"DL \", train_r2_dl,  test_r2_dl,  hold_r2_dl , round(dl_elapsed,3) ],\n",
    "           ]\n",
    "  h2o.display.H2ODisplay(table,header)\n",
    "  # --------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data has 5 columns and 83820 rows, test has 41603 rows, holdout has 13838\n",
      "\n",
      "gbm Model Build Progress: [##################################################] 100%\n",
      "\n",
      "drf Model Build Progress: [##################################################] 100%\n",
      "\n",
      "glm Model Build Progress: [##################################################] 100%\n",
      "\n",
      "deeplearning Model Build Progress: [##################################################] 100%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>Model</b></td>\n",
       "<td><b>R2 TRAIN</b></td>\n",
       "<td><b>R2 TEST</b></td>\n",
       "<td><b>R2 HOLDOUT</b></td>\n",
       "<td><b>Model Training Time (s)</b></td></tr>\n",
       "<tr><td>GBM</td>\n",
       "<td>0.9687211</td>\n",
       "<td>0.9200211</td>\n",
       "<td>0.9226079</td>\n",
       "<td>19.092</td></tr>\n",
       "<tr><td>DRF</td>\n",
       "<td>0.8570970</td>\n",
       "<td>0.8269861</td>\n",
       "<td>0.8256906</td>\n",
       "<td>20.128</td></tr>\n",
       "<tr><td>GLM</td>\n",
       "<td>0.7869223</td>\n",
       "<td>0.7810472</td>\n",
       "<td>0.7824196</td>\n",
       "<td>0.699</td></tr>\n",
       "<tr><td>DL </td>\n",
       "<td>0.8783443</td>\n",
       "<td>0.8570735</td>\n",
       "<td>0.8638957</td>\n",
       "<td>65.437</td></tr></table></div>"
      ],
      "text/plain": [
       "Model      R2 TRAIN    R2 TEST    R2 HOLDOUT    Model Training Time (s)\n",
       "-------  ----------  ---------  ------------  -------------------------\n",
       "GBM        0.968721   0.920021      0.922608                     19.092\n",
       "DRF        0.857097   0.826986      0.825691                     20.128\n",
       "GLM        0.786922   0.781047      0.78242                       0.699\n",
       "DL         0.878344   0.857074      0.863896                     65.437"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Split the data (into test & train), fit some models and predict on the holdout data\n",
    "split_fit_predict(bpd)\n",
    "# Here we see an r^2 of 0.91 for GBM, and 0.71 for GLM.  This means given just\n",
    "# the station, the month, and the day-of-week we can predict 90% of the\n",
    "# variance of the bike-trip-starts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Parse Progress: [##################################################] 100%\n",
      "Rows:17,520 Cols:50\n",
      "\n",
      "Chunk compression summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>chunk_type</b></td>\n",
       "<td><b>chunk_name</b></td>\n",
       "<td><b>count</b></td>\n",
       "<td><b>count_percentage</b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>size_percentage</b></td></tr>\n",
       "<tr><td>C0L</td>\n",
       "<td>Constant Integers</td>\n",
       "<td>107</td>\n",
       "<td>6.294118</td>\n",
       "<td>    8.4 KB</td>\n",
       "<td>0.7889721</td></tr>\n",
       "<tr><td>C0D</td>\n",
       "<td>Constant Reals</td>\n",
       "<td>436</td>\n",
       "<td>25.647058</td>\n",
       "<td>   34.1 KB</td>\n",
       "<td>3.2148771</td></tr>\n",
       "<tr><td>CXI</td>\n",
       "<td>Sparse Integers</td>\n",
       "<td>17</td>\n",
       "<td>1.0</td>\n",
       "<td>    1.5 KB</td>\n",
       "<td>0.1399135</td></tr>\n",
       "<tr><td>C1</td>\n",
       "<td>1-Byte Integers</td>\n",
       "<td>346</td>\n",
       "<td>20.352942</td>\n",
       "<td>  197.4 KB</td>\n",
       "<td>18.634672</td></tr>\n",
       "<tr><td>C1N</td>\n",
       "<td>1-Byte Integers (w/o NAs)</td>\n",
       "<td>214</td>\n",
       "<td>12.588236</td>\n",
       "<td>  122.3 KB</td>\n",
       "<td>11.544063</td></tr>\n",
       "<tr><td>C1S</td>\n",
       "<td>1-Byte Fractions</td>\n",
       "<td>214</td>\n",
       "<td>12.588236</td>\n",
       "<td>  125.3 KB</td>\n",
       "<td>11.822968</td></tr>\n",
       "<tr><td>C2S</td>\n",
       "<td>2-Byte Fractions</td>\n",
       "<td>196</td>\n",
       "<td>11.529412</td>\n",
       "<td>  214.5 KB</td>\n",
       "<td>20.242111</td></tr>\n",
       "<tr><td>C4S</td>\n",
       "<td>4-Byte Fractions</td>\n",
       "<td>170</td>\n",
       "<td>10.0</td>\n",
       "<td>  356.1 KB</td>\n",
       "<td>33.612423</td></tr></table></div>"
      ],
      "text/plain": [
       "chunk_type    chunk_name                 count    count_percentage    size      size_percentage\n",
       "------------  -------------------------  -------  ------------------  --------  -----------------\n",
       "C0L           Constant Integers          107      6.29412             8.4 KB    0.788972\n",
       "C0D           Constant Reals             436      25.6471             34.1 KB   3.21488\n",
       "CXI           Sparse Integers            17       1                   1.5 KB    0.139914\n",
       "C1            1-Byte Integers            346      20.3529             197.4 KB  18.6347\n",
       "C1N           1-Byte Integers (w/o NAs)  214      12.5882             122.3 KB  11.5441\n",
       "C1S           1-Byte Fractions           214      12.5882             125.3 KB  11.823\n",
       "C2S           2-Byte Fractions           196      11.5294             214.5 KB  20.2421\n",
       "C4S           4-Byte Fractions           170      10                  356.1 KB  33.6124"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Frame distribution summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>number_of_rows</b></td>\n",
       "<td><b>number_of_chunks_per_column</b></td>\n",
       "<td><b>number_of_chunks</b></td></tr>\n",
       "<tr><td>172.16.2.61:54321</td>\n",
       "<td>    1.0 MB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>1700.0</td></tr>\n",
       "<tr><td>mean</td>\n",
       "<td>    1.0 MB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>1700.0</td></tr>\n",
       "<tr><td>min</td>\n",
       "<td>    1.0 MB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>1700.0</td></tr>\n",
       "<tr><td>max</td>\n",
       "<td>    1.0 MB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>1700.0</td></tr>\n",
       "<tr><td>stddev</td>\n",
       "<td>      0  B</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>total</td>\n",
       "<td>    1.0 MB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>1700.0</td></tr></table></div>"
      ],
      "text/plain": [
       "                   size    number_of_rows    number_of_chunks_per_column    number_of_chunks\n",
       "-----------------  ------  ----------------  -----------------------------  ------------------\n",
       "172.16.2.61:54321  1.0 MB  17520             34                             1700\n",
       "mean               1.0 MB  17520             34                             1700\n",
       "min                1.0 MB  17520             34                             1700\n",
       "max                1.0 MB  17520             34                             1700\n",
       "stddev             0  B    0                 0                              0\n",
       "total              1.0 MB  17520             34                             1700"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>       </th><th>Year Local    </th><th>Month Local  </th><th>Day Local    </th><th>Hour Local   </th><th>Year UTC      </th><th>Month UTC    </th><th>Day UTC      </th><th>Hour UTC     </th><th>Cavok Reported  </th><th>Cloud Ceiling (m)  </th><th>Cloud Cover Fraction  </th><th>Cloud Cover Fraction 1  </th><th>Cloud Cover Fraction 2  </th><th>Cloud Cover Fraction 3  </th><th>Cloud Cover Fraction 4  </th><th>Cloud Cover Fraction 5  </th><th>Cloud Cover Fraction 6  </th><th>Cloud Height (m) 1  </th><th>Cloud Height (m) 2  </th><th>Cloud Height (m) 3  </th><th>Cloud Height (m) 4  </th><th>Cloud Height (m) 5  </th><th>Cloud Height (m) 6  </th><th>Dew Point (C)  </th><th>Humidity Fraction  </th><th>Precipitation One Hour (mm)  </th><th>Pressure Altimeter (mbar)  </th><th>Pressure Sea Level (mbar)  </th><th>Pressure Station (mbar)  </th><th>Snow Depth (cm)  </th><th>Temperature (C)  </th><th>Visibility (km)  </th><th>Weather Code 1  </th><th>Weather Code 1/ Description  </th><th>Weather Code 2  </th><th>Weather Code 2/ Description  </th><th>Weather Code 3  </th><th>Weather Code 3/ Description  </th><th>Weather Code 4  </th><th>Weather Code 4/ Description  </th><th>Weather Code 5  </th><th>Weather Code 5/ Description  </th><th>Weather Code 6  </th><th>Weather Code 6/ Description  </th><th>Weather Code Most Severe / Icon Code  </th><th>Weather Code Most Severe  </th><th>Weather Code Most Severe / Description  </th><th>Wind Direction (degrees)  </th><th>Wind Gust (m/s)  </th><th>Wind Speed (m/s)  </th></tr>\n",
       "<tr><td>type   </td><td>int           </td><td>int          </td><td>int          </td><td>int          </td><td>int           </td><td>int          </td><td>int          </td><td>int          </td><td>int             </td><td>real               </td><td>real                  </td><td>real                    </td><td>real                    </td><td>real                    </td><td>int                     </td><td>int                     </td><td>int                     </td><td>real                </td><td>real                </td><td>real                </td><td>int                 </td><td>int                 </td><td>int                 </td><td>real           </td><td>real               </td><td>real                         </td><td>real                       </td><td>int                        </td><td>int                      </td><td>int              </td><td>real             </td><td>real             </td><td>int             </td><td>enum                         </td><td>int             </td><td>enum                         </td><td>int             </td><td>enum                         </td><td>int             </td><td>enum                         </td><td>int             </td><td>enum                         </td><td>int             </td><td>enum                         </td><td>int                                   </td><td>int                       </td><td>enum                                    </td><td>int                       </td><td>real             </td><td>real              </td></tr>\n",
       "<tr><td>mins   </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>0.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>0.0          </td><td>0.0             </td><td>61.0               </td><td>0.0                   </td><td>0.0                     </td><td>0.25                    </td><td>0.5                     </td><td>NaN                     </td><td>NaN                     </td><td>NaN                     </td><td>60.96               </td><td>213.36              </td><td>365.76              </td><td>NaN                 </td><td>NaN                 </td><td>NaN                 </td><td>-26.7          </td><td>0.1251             </td><td>0.0                          </td><td>983.2949                   </td><td>NaN                        </td><td>NaN                      </td><td>NaN              </td><td>-15.6            </td><td>0.001            </td><td>1.0             </td><td>0.0                          </td><td>1.0             </td><td>0.0                          </td><td>1.0             </td><td>0.0                          </td><td>1.0             </td><td>0.0                          </td><td>1.0             </td><td>0.0                          </td><td>3.0             </td><td>0.0                          </td><td>0.0                                   </td><td>1.0                       </td><td>0.0                                     </td><td>10.0                      </td><td>7.2              </td><td>0.0               </td></tr>\n",
       "<tr><td>mean   </td><td>2013.5        </td><td>6.52602739726</td><td>15.7205479452</td><td>11.5         </td><td>2013.50057078 </td><td>6.52511415525</td><td>15.721347032 </td><td>11.5001141553</td><td>0.0             </td><td>1306.31195846      </td><td>0.416742490522        </td><td>0.361207349081          </td><td>0.872445384073          </td><td>0.963045685279          </td><td>0.0                     </td><td>0.0                     </td><td>0.0                     </td><td>1293.9822682        </td><td>1643.73900166       </td><td>2084.89386376       </td><td>0.0                 </td><td>0.0                 </td><td>0.0                 </td><td>4.31304646766  </td><td>0.596736389159     </td><td>1.37993010753                </td><td>1017.82581441              </td><td>0.0                        </td><td>0.0                      </td><td>0.0              </td><td>12.5789090701    </td><td>14.3914429682    </td><td>4.84251968504   </td><td>NaN                          </td><td>3.65867689358   </td><td>NaN                          </td><td>2.84660766962   </td><td>NaN                          </td><td>2.01149425287   </td><td>NaN                          </td><td>4.125           </td><td>NaN                          </td><td>3.0             </td><td>0.0                          </td><td>1.37848173516                         </td><td>4.84251968504             </td><td>NaN                                     </td><td>194.69525682              </td><td>9.42216948073    </td><td>2.41032887849     </td></tr>\n",
       "<tr><td>maxs   </td><td>2014.0        </td><td>12.0         </td><td>31.0         </td><td>23.0         </td><td>2015.0        </td><td>12.0         </td><td>31.0         </td><td>23.0         </td><td>0.0             </td><td>3657.6             </td><td>1.0                   </td><td>1.0                     </td><td>1.0                     </td><td>1.0                     </td><td>NaN                     </td><td>NaN                     </td><td>NaN                     </td><td>3657.5999           </td><td>3657.5999           </td><td>3657.5999           </td><td>NaN                 </td><td>NaN                 </td><td>NaN                 </td><td>24.4           </td><td>1.0                </td><td>26.924                       </td><td>1042.2113                  </td><td>NaN                        </td><td>NaN                      </td><td>NaN              </td><td>36.1             </td><td>16.0934          </td><td>60.0            </td><td>11.0                         </td><td>60.0            </td><td>10.0                         </td><td>36.0            </td><td>7.0                          </td><td>27.0            </td><td>4.0                          </td><td>27.0            </td><td>2.0                          </td><td>3.0             </td><td>0.0                          </td><td>16.0                                  </td><td>60.0                      </td><td>11.0                                    </td><td>360.0                     </td><td>20.58            </td><td>10.8              </td></tr>\n",
       "<tr><td>sigma  </td><td>0.500014270017</td><td>3.44794972385</td><td>8.79649804852</td><td>6.92238411188</td><td>0.500584411716</td><td>3.44782405458</td><td>8.79561488868</td><td>6.92230165203</td><td>0.0             </td><td>995.339856966      </td><td>0.462720830993        </td><td>0.42770569708           </td><td>0.197155690367          </td><td>0.0861015598104         </td><td>-0.0                    </td><td>-0.0                    </td><td>-0.0                    </td><td>962.743095854       </td><td>916.73861349        </td><td>887.215847511       </td><td>-0.0                </td><td>-0.0                </td><td>-0.0                </td><td>10.9731282097  </td><td>0.185792011866     </td><td>2.56215129179                </td><td>7.46451697179              </td><td>-0.0                       </td><td>-0.0                     </td><td>-0.0             </td><td>10.0396739531    </td><td>3.69893623033    </td><td>5.70486576983   </td><td>NaN                          </td><td>6.13386253912   </td><td>NaN                          </td><td>5.80553286364   </td><td>NaN                          </td><td>3.12340844261   </td><td>NaN                          </td><td>6.15223536611   </td><td>NaN                          </td><td>0.0             </td><td>0.0                          </td><td>4.07386062702                         </td><td>5.70486576983             </td><td>NaN                                     </td><td>106.350000031             </td><td>1.81511871115    </td><td>1.61469790524     </td></tr>\n",
       "<tr><td>zeros  </td><td>0             </td><td>0            </td><td>0            </td><td>730          </td><td>0             </td><td>0            </td><td>0            </td><td>730          </td><td>17455           </td><td>0                  </td><td>8758                  </td><td>8758                    </td><td>0                       </td><td>0                       </td><td>-17520                  </td><td>-17520                  </td><td>-17520                  </td><td>0                   </td><td>0                   </td><td>0                   </td><td>-17520              </td><td>-17520              </td><td>-17520              </td><td>268            </td><td>0                  </td><td>501                          </td><td>0                          </td><td>-17520                     </td><td>-17520                   </td><td>-17520           </td><td>269              </td><td>0                </td><td>0               </td><td>17                           </td><td>0               </td><td>30                           </td><td>0               </td><td>13                           </td><td>-5044           </td><td>-5024                        </td><td>-11241          </td><td>-11229                       </td><td>-17030          </td><td>-17028                       </td><td>14980                                 </td><td>0                         </td><td>17                                      </td><td>0                         </td><td>0                </td><td>2768              </td></tr>\n",
       "<tr><td>missing</td><td>0             </td><td>0            </td><td>0            </td><td>0            </td><td>0             </td><td>0            </td><td>0            </td><td>0            </td><td>65              </td><td>10780              </td><td>375                   </td><td>375                     </td><td>14682                   </td><td>16535                   </td><td>17520                   </td><td>17520                   </td><td>17520                   </td><td>9103                </td><td>14683               </td><td>16535               </td><td>17520               </td><td>17520               </td><td>17520               </td><td>67             </td><td>67                 </td><td>15660                        </td><td>360                        </td><td>17520                      </td><td>17520                    </td><td>17520            </td><td>67               </td><td>412              </td><td>14980           </td><td>14980                        </td><td>16477           </td><td>16477                        </td><td>17181           </td><td>17181                        </td><td>17433           </td><td>17433                        </td><td>17504           </td><td>17504                        </td><td>17518           </td><td>17518                        </td><td>0                                     </td><td>14980                     </td><td>14980                                   </td><td>9382                      </td><td>14381            </td><td>1283              </td></tr>\n",
       "<tr><td>0      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>0.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>5.0          </td><td>0.0             </td><td>2895.6             </td><td>1.0                   </td><td>0.9                     </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>2895.5999           </td><td>3352.8              </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-5.0           </td><td>0.5447             </td><td>nan                          </td><td>1013.0917                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.3              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>nan                       </td><td>nan              </td><td>2.57              </td></tr>\n",
       "<tr><td>1      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>1.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>6.0          </td><td>0.0             </td><td>3048.0             </td><td>1.0                   </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>3048.0              </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-4.4           </td><td>0.5463             </td><td>nan                          </td><td>1012.0759                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.9              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>260.0                     </td><td>9.77             </td><td>4.63              </td></tr>\n",
       "<tr><td>2      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>2.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>7.0          </td><td>0.0             </td><td>1828.8             </td><td>1.0                   </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>1828.7999           </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-3.3           </td><td>0.619              </td><td>nan                          </td><td>1012.4145                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.3              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>nan                       </td><td>7.72             </td><td>1.54              </td></tr>\n",
       "<tr><td>3      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>3.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>8.0          </td><td>0.0             </td><td>1463.0             </td><td>1.0                   </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>1463.04             </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-2.8           </td><td>0.6159             </td><td>nan                          </td><td>1012.4145                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.9              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>nan                       </td><td>nan              </td><td>3.09              </td></tr>\n",
       "<tr><td>4      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>4.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>9.0          </td><td>0.0             </td><td>1402.1             </td><td>1.0                   </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>1402.08             </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-2.8           </td><td>0.6159             </td><td>nan                          </td><td>1012.7531                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.9              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>260.0                     </td><td>nan              </td><td>4.12              </td></tr>\n",
       "<tr><td>5      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>5.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>10.0         </td><td>0.0             </td><td>1524.0             </td><td>1.0                   </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>1524.0              </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-2.8           </td><td>0.6159             </td><td>nan                          </td><td>1012.4145                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.9              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>nan                       </td><td>nan              </td><td>3.09              </td></tr>\n",
       "<tr><td>6      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>6.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>11.0         </td><td>0.0             </td><td>1524.0             </td><td>1.0                   </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>1524.0              </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-3.3           </td><td>0.5934             </td><td>nan                          </td><td>1012.0759                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.9              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>nan                       </td><td>9.26             </td><td>3.09              </td></tr>\n",
       "<tr><td>7      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>7.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>12.0         </td><td>0.0             </td><td>1524.0             </td><td>1.0                   </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>1524.0              </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-3.3           </td><td>0.5934             </td><td>nan                          </td><td>1012.4145                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.9              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>260.0                     </td><td>9.26             </td><td>4.63              </td></tr>\n",
       "<tr><td>8      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>8.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>13.0         </td><td>0.0             </td><td>1524.0             </td><td>1.0                   </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>1524.0              </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-2.8           </td><td>0.6425             </td><td>nan                          </td><td>1012.4145                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.3              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>260.0                     </td><td>nan              </td><td>3.09              </td></tr>\n",
       "<tr><td>9      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>9.0          </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>14.0         </td><td>0.0             </td><td>1524.0             </td><td>1.0                   </td><td>0.9                     </td><td>1.0                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>nan                     </td><td>1524.0              </td><td>3657.5999           </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>nan                 </td><td>-2.8           </td><td>0.6159             </td><td>nan                          </td><td>1012.4145                  </td><td>nan                        </td><td>nan                      </td><td>nan              </td><td>3.9              </td><td>16.0934          </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>nan             </td><td>                             </td><td>0.0                                   </td><td>nan                       </td><td>                                        </td><td>nan                       </td><td>9.26             </td><td>3.09              </td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ----------\n",
    "# 5- Now lets add some weather\n",
    "# Load weather data\n",
    "wthr1 = h2o.import_file(path=[mylocate(\"bigdata/laptop/citibike-nyc/31081_New_York_City__Hourly_2013.csv\"),\n",
    "                               mylocate(\"bigdata/laptop/citibike-nyc/31081_New_York_City__Hourly_2014.csv\")])\n",
    "# Peek at the data\n",
    "wthr1.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows:17,520 Cols:9\n",
      "\n",
      "Chunk compression summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>chunk_type</b></td>\n",
       "<td><b>chunk_name</b></td>\n",
       "<td><b>count</b></td>\n",
       "<td><b>count_percentage</b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>size_percentage</b></td></tr>\n",
       "<tr><td>C0L</td>\n",
       "<td>Constant Integers</td>\n",
       "<td>46</td>\n",
       "<td>15.0326805</td>\n",
       "<td>    3.6 KB</td>\n",
       "<td>1.780005</td></tr>\n",
       "<tr><td>C1</td>\n",
       "<td>1-Byte Integers</td>\n",
       "<td>34</td>\n",
       "<td>11.111112</td>\n",
       "<td>   19.4 KB</td>\n",
       "<td>9.592678</td></tr>\n",
       "<tr><td>C1N</td>\n",
       "<td>1-Byte Integers (w/o NAs)</td>\n",
       "<td>90</td>\n",
       "<td>29.411766</td>\n",
       "<td>   51.5 KB</td>\n",
       "<td>25.494701</td></tr>\n",
       "<tr><td>C1S</td>\n",
       "<td>1-Byte Fractions</td>\n",
       "<td>42</td>\n",
       "<td>13.725491</td>\n",
       "<td>   24.0 KB</td>\n",
       "<td>11.894592</td></tr>\n",
       "<tr><td>C2S</td>\n",
       "<td>2-Byte Fractions</td>\n",
       "<td>94</td>\n",
       "<td>30.718956</td>\n",
       "<td>  103.4 KB</td>\n",
       "<td>51.238026</td></tr></table></div>"
      ],
      "text/plain": [
       "chunk_type    chunk_name                 count    count_percentage    size      size_percentage\n",
       "------------  -------------------------  -------  ------------------  --------  -----------------\n",
       "C0L           Constant Integers          46       15.0327             3.6 KB    1.78001\n",
       "C1            1-Byte Integers            34       11.1111             19.4 KB   9.59268\n",
       "C1N           1-Byte Integers (w/o NAs)  90       29.4118             51.5 KB   25.4947\n",
       "C1S           1-Byte Fractions           42       13.7255             24.0 KB   11.8946\n",
       "C2S           2-Byte Fractions           94       30.719              103.4 KB  51.238"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Frame distribution summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>number_of_rows</b></td>\n",
       "<td><b>number_of_chunks_per_column</b></td>\n",
       "<td><b>number_of_chunks</b></td></tr>\n",
       "<tr><td>172.16.2.61:54321</td>\n",
       "<td>  201.9 KB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>306.0</td></tr>\n",
       "<tr><td>mean</td>\n",
       "<td>  201.9 KB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>306.0</td></tr>\n",
       "<tr><td>min</td>\n",
       "<td>  201.9 KB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>306.0</td></tr>\n",
       "<tr><td>max</td>\n",
       "<td>  201.9 KB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>306.0</td></tr>\n",
       "<tr><td>stddev</td>\n",
       "<td>      0  B</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>total</td>\n",
       "<td>  201.9 KB</td>\n",
       "<td>17520.0</td>\n",
       "<td>34.0</td>\n",
       "<td>306.0</td></tr></table></div>"
      ],
      "text/plain": [
       "                   size      number_of_rows    number_of_chunks_per_column    number_of_chunks\n",
       "-----------------  --------  ----------------  -----------------------------  ------------------\n",
       "172.16.2.61:54321  201.9 KB  17520             34                             306\n",
       "mean               201.9 KB  17520             34                             306\n",
       "min                201.9 KB  17520             34                             306\n",
       "max                201.9 KB  17520             34                             306\n",
       "stddev             0  B      0                 0                              0\n",
       "total              201.9 KB  17520             34                             306"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>       </th><th>Year Local    </th><th>Month Local  </th><th>Day Local    </th><th>Hour Local   </th><th>Dew Point (C)  </th><th>Humidity Fraction  </th><th>Rain (mm)    </th><th>Temperature (C)  </th><th>WC1  </th></tr>\n",
       "<tr><td>type   </td><td>int           </td><td>int          </td><td>int          </td><td>int          </td><td>real           </td><td>real               </td><td>real         </td><td>real             </td><td>enum </td></tr>\n",
       "<tr><td>mins   </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>0.0          </td><td>-26.7          </td><td>0.1251             </td><td>0.0          </td><td>-15.6            </td><td>0.0  </td></tr>\n",
       "<tr><td>mean   </td><td>2013.5        </td><td>6.52602739726</td><td>15.7205479452</td><td>11.5         </td><td>4.31304646766  </td><td>0.596736389159     </td><td>1.37993010753</td><td>12.5789090701    </td><td>NaN  </td></tr>\n",
       "<tr><td>maxs   </td><td>2014.0        </td><td>12.0         </td><td>31.0         </td><td>23.0         </td><td>24.4           </td><td>1.0                </td><td>26.924       </td><td>36.1             </td><td>11.0 </td></tr>\n",
       "<tr><td>sigma  </td><td>0.500014270017</td><td>3.44794972385</td><td>8.79649804852</td><td>6.92238411188</td><td>10.9731282097  </td><td>0.185792011866     </td><td>2.56215129179</td><td>10.0396739531    </td><td>NaN  </td></tr>\n",
       "<tr><td>zeros  </td><td>0             </td><td>0            </td><td>0            </td><td>730          </td><td>268            </td><td>0                  </td><td>501          </td><td>269              </td><td>17   </td></tr>\n",
       "<tr><td>missing</td><td>0             </td><td>0            </td><td>0            </td><td>0            </td><td>67             </td><td>67                 </td><td>15660        </td><td>67               </td><td>14980</td></tr>\n",
       "<tr><td>0      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>0.0          </td><td>-5.0           </td><td>0.5447             </td><td>nan          </td><td>3.3              </td><td>     </td></tr>\n",
       "<tr><td>1      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>1.0          </td><td>-4.4           </td><td>0.5463             </td><td>nan          </td><td>3.9              </td><td>     </td></tr>\n",
       "<tr><td>2      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>2.0          </td><td>-3.3           </td><td>0.619              </td><td>nan          </td><td>3.3              </td><td>     </td></tr>\n",
       "<tr><td>3      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>3.0          </td><td>-2.8           </td><td>0.6159             </td><td>nan          </td><td>3.9              </td><td>     </td></tr>\n",
       "<tr><td>4      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>4.0          </td><td>-2.8           </td><td>0.6159             </td><td>nan          </td><td>3.9              </td><td>     </td></tr>\n",
       "<tr><td>5      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>5.0          </td><td>-2.8           </td><td>0.6159             </td><td>nan          </td><td>3.9              </td><td>     </td></tr>\n",
       "<tr><td>6      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>6.0          </td><td>-3.3           </td><td>0.5934             </td><td>nan          </td><td>3.9              </td><td>     </td></tr>\n",
       "<tr><td>7      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>7.0          </td><td>-3.3           </td><td>0.5934             </td><td>nan          </td><td>3.9              </td><td>     </td></tr>\n",
       "<tr><td>8      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>8.0          </td><td>-2.8           </td><td>0.6425             </td><td>nan          </td><td>3.3              </td><td>     </td></tr>\n",
       "<tr><td>9      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>9.0          </td><td>-2.8           </td><td>0.6159             </td><td>nan          </td><td>3.9              </td><td>     </td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Lots of columns in there!  Lets plan on converting to time-since-epoch to do\n",
    "# a 'join' with the bike data, plus gather weather info that might affect\n",
    "# cyclists - rain, snow, temperature.  Alas, drop the \"snow\" column since it's\n",
    "# all NA's.  Also add in dew point and humidity just in case.  Slice out just\n",
    "# the columns of interest and drop the rest.\n",
    "wthr2 = wthr1[[\"Year Local\",\"Month Local\",\"Day Local\",\"Hour Local\",\"Dew Point (C)\",\"Humidity Fraction\",\"Precipitation One Hour (mm)\",\"Temperature (C)\",\"Weather Code 1/ Description\"]]\n",
    "\n",
    "wthr2.set_name(wthr2.names.index(\"Precipitation One Hour (mm)\"), \"Rain (mm)\")\n",
    "wthr2.set_name(wthr2.names.index(\"Weather Code 1/ Description\"), \"WC1\")\n",
    "wthr2.describe()\n",
    "# Much better!  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Filter down to the weather at Noon\n",
    "wthr3 = wthr2[ wthr2[\"Hour Local\"]==12 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows:730 Cols:11\n",
      "\n",
      "Chunk compression summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>chunk_type</b></td>\n",
       "<td><b>chunk_name</b></td>\n",
       "<td><b>count</b></td>\n",
       "<td><b>count_percentage</b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>size_percentage</b></td></tr>\n",
       "<tr><td>C0L</td>\n",
       "<td>Constant Integers</td>\n",
       "<td>80</td>\n",
       "<td>21.390373</td>\n",
       "<td>    6.3 KB</td>\n",
       "<td>12.498779</td></tr>\n",
       "<tr><td>C0D</td>\n",
       "<td>Constant Reals</td>\n",
       "<td>13</td>\n",
       "<td>3.4759357</td>\n",
       "<td>    1.0 KB</td>\n",
       "<td>2.0310516</td></tr>\n",
       "<tr><td>C1</td>\n",
       "<td>1-Byte Integers</td>\n",
       "<td>30</td>\n",
       "<td>8.021391</td>\n",
       "<td>    2.6 KB</td>\n",
       "<td>5.2455816</td></tr>\n",
       "<tr><td>C1N</td>\n",
       "<td>1-Byte Integers (w/o NAs)</td>\n",
       "<td>56</td>\n",
       "<td>14.973262</td>\n",
       "<td>    4.9 KB</td>\n",
       "<td>9.801778</td></tr>\n",
       "<tr><td>C1S</td>\n",
       "<td>1-Byte Fractions</td>\n",
       "<td>34</td>\n",
       "<td>9.090909</td>\n",
       "<td>    3.5 KB</td>\n",
       "<td>7.0032225</td></tr>\n",
       "<tr><td>C2S</td>\n",
       "<td>2-Byte Fractions</td>\n",
       "<td>34</td>\n",
       "<td>9.090909</td>\n",
       "<td>    4.2 KB</td>\n",
       "<td>8.4288645</td></tr>\n",
       "<tr><td>CUD</td>\n",
       "<td>Unique Reals</td>\n",
       "<td>25</td>\n",
       "<td>6.6844916</td>\n",
       "<td>    3.6 KB</td>\n",
       "<td>7.2297626</td></tr>\n",
       "<tr><td>C8D</td>\n",
       "<td>64-bit Reals</td>\n",
       "<td>102</td>\n",
       "<td>27.272728</td>\n",
       "<td>   23.9 KB</td>\n",
       "<td>47.76096</td></tr></table></div>"
      ],
      "text/plain": [
       "chunk_type    chunk_name                 count    count_percentage    size     size_percentage\n",
       "------------  -------------------------  -------  ------------------  -------  -----------------\n",
       "C0L           Constant Integers          80       21.3904             6.3 KB   12.4988\n",
       "C0D           Constant Reals             13       3.47594             1.0 KB   2.03105\n",
       "C1            1-Byte Integers            30       8.02139             2.6 KB   5.24558\n",
       "C1N           1-Byte Integers (w/o NAs)  56       14.9733             4.9 KB   9.80178\n",
       "C1S           1-Byte Fractions           34       9.09091             3.5 KB   7.00322\n",
       "C2S           2-Byte Fractions           34       9.09091             4.2 KB   8.42886\n",
       "CUD           Unique Reals               25       6.68449             3.6 KB   7.22976\n",
       "C8D           64-bit Reals               102      27.2727             23.9 KB  47.761"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Frame distribution summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>number_of_rows</b></td>\n",
       "<td><b>number_of_chunks_per_column</b></td>\n",
       "<td><b>number_of_chunks</b></td></tr>\n",
       "<tr><td>172.16.2.61:54321</td>\n",
       "<td>   50.0 KB</td>\n",
       "<td>730.0</td>\n",
       "<td>34.0</td>\n",
       "<td>374.0</td></tr>\n",
       "<tr><td>mean</td>\n",
       "<td>   50.0 KB</td>\n",
       "<td>730.0</td>\n",
       "<td>34.0</td>\n",
       "<td>374.0</td></tr>\n",
       "<tr><td>min</td>\n",
       "<td>   50.0 KB</td>\n",
       "<td>730.0</td>\n",
       "<td>34.0</td>\n",
       "<td>374.0</td></tr>\n",
       "<tr><td>max</td>\n",
       "<td>   50.0 KB</td>\n",
       "<td>730.0</td>\n",
       "<td>34.0</td>\n",
       "<td>374.0</td></tr>\n",
       "<tr><td>stddev</td>\n",
       "<td>      0  B</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>total</td>\n",
       "<td>   50.0 KB</td>\n",
       "<td>730.0</td>\n",
       "<td>34.0</td>\n",
       "<td>374.0</td></tr></table></div>"
      ],
      "text/plain": [
       "                   size     number_of_rows    number_of_chunks_per_column    number_of_chunks\n",
       "-----------------  -------  ----------------  -----------------------------  ------------------\n",
       "172.16.2.61:54321  50.0 KB  730               34                             374\n",
       "mean               50.0 KB  730               34                             374\n",
       "min                50.0 KB  730               34                             374\n",
       "max                50.0 KB  730               34                             374\n",
       "stddev             0  B     0                 0                              0\n",
       "total              50.0 KB  730               34                             374"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>       </th><th>Year Local    </th><th>Month Local  </th><th>Day Local    </th><th>Hour Local  </th><th>Dew Point (C)  </th><th>Humidity Fraction  </th><th>Rain (mm)    </th><th>Temperature (C)  </th><th>WC1  </th><th>msec            </th><th>Days         </th></tr>\n",
       "<tr><td>type   </td><td>int           </td><td>int          </td><td>int          </td><td>int         </td><td>real           </td><td>real               </td><td>real         </td><td>real             </td><td>enum </td><td>int             </td><td>int          </td></tr>\n",
       "<tr><td>mins   </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>12.0        </td><td>-26.7          </td><td>0.1723             </td><td>0.0          </td><td>-13.9            </td><td>0.0  </td><td>1.3570704e+12   </td><td>15706.0      </td></tr>\n",
       "<tr><td>mean   </td><td>2013.5        </td><td>6.52602739726</td><td>15.7205479452</td><td>12.0        </td><td>4.23012379642  </td><td>0.539728198074     </td><td>1.53125714286</td><td>14.0687757909    </td><td>NaN  </td><td>1.3885608526e+12</td><td>16070.5      </td></tr>\n",
       "<tr><td>maxs   </td><td>2014.0        </td><td>12.0         </td><td>31.0         </td><td>12.0        </td><td>23.3           </td><td>1.0                </td><td>12.446       </td><td>34.4             </td><td>10.0 </td><td>1.420056e+12    </td><td>16435.0      </td></tr>\n",
       "<tr><td>sigma  </td><td>0.500342818004</td><td>3.45021529307</td><td>8.80227802701</td><td>0.0         </td><td>11.1062964725  </td><td>0.179945027923     </td><td>2.36064248615</td><td>10.3989855149    </td><td>NaN  </td><td>18219740080.4   </td><td>210.877136425</td></tr>\n",
       "<tr><td>zeros  </td><td>0             </td><td>0            </td><td>0            </td><td>0           </td><td>14             </td><td>0                  </td><td>-174         </td><td>7                </td><td>-83  </td><td>0               </td><td>0            </td></tr>\n",
       "<tr><td>missing</td><td>0             </td><td>0            </td><td>0            </td><td>0           </td><td>3              </td><td>3                  </td><td>660          </td><td>3                </td><td>620  </td><td>0               </td><td>0            </td></tr>\n",
       "<tr><td>0      </td><td>2013.0        </td><td>1.0          </td><td>1.0          </td><td>12.0        </td><td>-3.3           </td><td>0.5934             </td><td>nan          </td><td>3.9              </td><td>     </td><td>1.3570704e+12   </td><td>15706.0      </td></tr>\n",
       "<tr><td>1      </td><td>2013.0        </td><td>1.0          </td><td>2.0          </td><td>12.0        </td><td>-11.7          </td><td>0.4806             </td><td>nan          </td><td>-2.2             </td><td>     </td><td>1.3571568e+12   </td><td>15707.0      </td></tr>\n",
       "<tr><td>2      </td><td>2013.0        </td><td>1.0          </td><td>3.0          </td><td>12.0        </td><td>-10.6          </td><td>0.5248             </td><td>nan          </td><td>-2.2             </td><td>     </td><td>1.3572432e+12   </td><td>15708.0      </td></tr>\n",
       "<tr><td>3      </td><td>2013.0        </td><td>1.0          </td><td>4.0          </td><td>12.0        </td><td>-7.2           </td><td>0.4976             </td><td>nan          </td><td>2.2              </td><td>     </td><td>1.3573296e+12   </td><td>15709.0      </td></tr>\n",
       "<tr><td>4      </td><td>2013.0        </td><td>1.0          </td><td>5.0          </td><td>12.0        </td><td>-7.2           </td><td>0.426              </td><td>nan          </td><td>4.4              </td><td>     </td><td>1.357416e+12    </td><td>15710.0      </td></tr>\n",
       "<tr><td>5      </td><td>2013.0        </td><td>1.0          </td><td>6.0          </td><td>12.0        </td><td>-1.7           </td><td>0.6451             </td><td>nan          </td><td>4.4              </td><td>haze </td><td>1.3575024e+12   </td><td>15711.0      </td></tr>\n",
       "<tr><td>6      </td><td>2013.0        </td><td>1.0          </td><td>7.0          </td><td>12.0        </td><td>-6.1           </td><td>0.4119             </td><td>nan          </td><td>6.1              </td><td>     </td><td>1.3575888e+12   </td><td>15712.0      </td></tr>\n",
       "<tr><td>7      </td><td>2013.0        </td><td>1.0          </td><td>8.0          </td><td>12.0        </td><td>-1.7           </td><td>0.5314             </td><td>nan          </td><td>7.2              </td><td>     </td><td>1.3576752e+12   </td><td>15713.0      </td></tr>\n",
       "<tr><td>8      </td><td>2013.0        </td><td>1.0          </td><td>9.0          </td><td>12.0        </td><td>0.6            </td><td>0.56               </td><td>nan          </td><td>8.9              </td><td>haze </td><td>1.3577616e+12   </td><td>15714.0      </td></tr>\n",
       "<tr><td>9      </td><td>2013.0        </td><td>1.0          </td><td>10.0         </td><td>12.0        </td><td>-6.1           </td><td>0.3952             </td><td>nan          </td><td>6.7              </td><td>     </td><td>1.357848e+12    </td><td>15715.0      </td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Lets now get Days since the epoch... we'll convert year/month/day into Epoch\n",
    "# time, and then back to Epoch days.  Need zero-based month and days, but have\n",
    "# 1-based.\n",
    "wthr3[\"msec\"] = h2o.H2OFrame.mktime(year=wthr3[\"Year Local\"], month=wthr3[\"Month Local\"]-1, day=wthr3[\"Day Local\"]-1, hour=wthr3[\"Hour Local\"])\n",
    "secsPerDay=1000*60*60*24\n",
    "wthr3[\"Days\"] = (wthr3[\"msec\"]/secsPerDay).floor()\n",
    "wthr3.describe()\n",
    "# msec looks sane (numbers like 1.3e12 are in the correct range for msec since\n",
    "# 1970).  Epoch Days matches closely with the epoch day numbers from the\n",
    "# CitiBike dataset.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Lets drop off the extra time columns to make a easy-to-handle dataset.\n",
    "wthr4 = wthr3.drop(\"Year Local\").drop(\"Month Local\").drop(\"Day Local\").drop(\"Hour Local\").drop(\"msec\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Also, most rain numbers are missing - lets assume those are zero rain days\n",
    "rain = wthr4[\"Rain (mm)\"]\n",
    "rain[ rain.isna() ] = 0\n",
    "wthr4[\"Rain (mm)\"] = rain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Merge Daily Weather with Bikes-Per-Day\n",
      "Rows:139,261 Cols:10\n",
      "\n",
      "Chunk compression summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>chunk_type</b></td>\n",
       "<td><b>chunk_name</b></td>\n",
       "<td><b>count</b></td>\n",
       "<td><b>count_percentage</b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>size_percentage</b></td></tr>\n",
       "<tr><td>C0L</td>\n",
       "<td>Constant Integers</td>\n",
       "<td>50</td>\n",
       "<td>15.625</td>\n",
       "<td>    3.9 KB</td>\n",
       "<td>0.2143667</td></tr>\n",
       "<tr><td>C0D</td>\n",
       "<td>Constant Reals</td>\n",
       "<td>6</td>\n",
       "<td>1.8750001</td>\n",
       "<td>    480  B</td>\n",
       "<td>0.0257240</td></tr>\n",
       "<tr><td>CBS</td>\n",
       "<td>Bits</td>\n",
       "<td>1</td>\n",
       "<td>0.3125</td>\n",
       "<td>    614  B</td>\n",
       "<td>0.0329053</td></tr>\n",
       "<tr><td>C1</td>\n",
       "<td>1-Byte Integers</td>\n",
       "<td>26</td>\n",
       "<td>8.125</td>\n",
       "<td>  112.2 KB</td>\n",
       "<td>6.1573634</td></tr>\n",
       "<tr><td>C1N</td>\n",
       "<td>1-Byte Integers (w/o NAs)</td>\n",
       "<td>47</td>\n",
       "<td>14.687499</td>\n",
       "<td>  202.9 KB</td>\n",
       "<td>11.132173</td></tr>\n",
       "<tr><td>C1S</td>\n",
       "<td>1-Byte Fractions</td>\n",
       "<td>32</td>\n",
       "<td>10.0</td>\n",
       "<td>  138.6 KB</td>\n",
       "<td>7.6072865</td></tr>\n",
       "<tr><td>C2</td>\n",
       "<td>2-Byte Integers</td>\n",
       "<td>62</td>\n",
       "<td>19.375</td>\n",
       "<td>  531.1 KB</td>\n",
       "<td>29.146162</td></tr>\n",
       "<tr><td>CUD</td>\n",
       "<td>Unique Reals</td>\n",
       "<td>96</td>\n",
       "<td>30.000002</td>\n",
       "<td>  832.5 KB</td>\n",
       "<td>45.684017</td></tr></table></div>"
      ],
      "text/plain": [
       "chunk_type    chunk_name                 count    count_percentage    size      size_percentage\n",
       "------------  -------------------------  -------  ------------------  --------  -----------------\n",
       "C0L           Constant Integers          50       15.625              3.9 KB    0.214367\n",
       "C0D           Constant Reals             6        1.875               480  B    0.025724\n",
       "CBS           Bits                       1        0.3125              614  B    0.0329053\n",
       "C1            1-Byte Integers            26       8.125               112.2 KB  6.15736\n",
       "C1N           1-Byte Integers (w/o NAs)  47       14.6875             202.9 KB  11.1322\n",
       "C1S           1-Byte Fractions           32       10                  138.6 KB  7.60729\n",
       "C2            2-Byte Integers            62       19.375              531.1 KB  29.1462\n",
       "CUD           Unique Reals               96       30                  832.5 KB  45.684"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Frame distribution summary:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>size</b></td>\n",
       "<td><b>number_of_rows</b></td>\n",
       "<td><b>number_of_chunks_per_column</b></td>\n",
       "<td><b>number_of_chunks</b></td></tr>\n",
       "<tr><td>172.16.2.61:54321</td>\n",
       "<td>    1.8 MB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>320.0</td></tr>\n",
       "<tr><td>mean</td>\n",
       "<td>    1.8 MB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>320.0</td></tr>\n",
       "<tr><td>min</td>\n",
       "<td>    1.8 MB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>320.0</td></tr>\n",
       "<tr><td>max</td>\n",
       "<td>    1.8 MB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>320.0</td></tr>\n",
       "<tr><td>stddev</td>\n",
       "<td>      0  B</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>total</td>\n",
       "<td>    1.8 MB</td>\n",
       "<td>139261.0</td>\n",
       "<td>32.0</td>\n",
       "<td>320.0</td></tr></table></div>"
      ],
      "text/plain": [
       "                   size    number_of_rows    number_of_chunks_per_column    number_of_chunks\n",
       "-----------------  ------  ----------------  -----------------------------  ------------------\n",
       "172.16.2.61:54321  1.8 MB  139261            32                             320\n",
       "mean               1.8 MB  139261            32                             320\n",
       "min                1.8 MB  139261            32                             320\n",
       "max                1.8 MB  139261            32                             320\n",
       "stddev             0  B    0                 0                              0\n",
       "total              1.8 MB  139261            32                             320"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>       </th><th>Days         </th><th>start station name  </th><th>bikes        </th><th>Month  </th><th>DayOfWeek  </th><th>Humidity Fraction  </th><th>Rain (mm)  </th><th>Temperature (C)  </th><th>WC1   </th><th>Dew Point (C)  </th></tr>\n",
       "<tr><td>type   </td><td>int          </td><td>enum                </td><td>int          </td><td>enum   </td><td>enum       </td><td>real               </td><td>int        </td><td>real             </td><td>enum  </td><td>real           </td></tr>\n",
       "<tr><td>mins   </td><td>15887.0      </td><td>0.0                 </td><td>1.0          </td><td>0.0    </td><td>0.0        </td><td>0.1723             </td><td>0.0        </td><td>-13.9            </td><td>0.0   </td><td>-26.7          </td></tr>\n",
       "<tr><td>mean   </td><td>16099.9758008</td><td>NaN                 </td><td>74.7329833909</td><td>NaN    </td><td>NaN        </td><td>0.532494425803     </td><td>0.0        </td><td>15.6334205959    </td><td>NaN   </td><td>5.47825137402  </td></tr>\n",
       "<tr><td>maxs   </td><td>16314.0      </td><td>339.0               </td><td>680.0        </td><td>11.0   </td><td>6.0        </td><td>1.0                </td><td>0.0        </td><td>34.4             </td><td>10.0  </td><td>23.3           </td></tr>\n",
       "<tr><td>sigma  </td><td>123.635133897</td><td>NaN                 </td><td>64.1241591283</td><td>NaN    </td><td>NaN        </td><td>0.178408938664     </td><td>0.0        </td><td>10.9454511961    </td><td>NaN   </td><td>11.7308194576  </td></tr>\n",
       "<tr><td>zeros  </td><td>0            </td><td>428                 </td><td>0            </td><td>9949   </td><td>19880      </td><td>0                  </td><td>139261     </td><td>1598             </td><td>-25811</td><td>1954           </td></tr>\n",
       "<tr><td>missing</td><td>0            </td><td>0                   </td><td>0            </td><td>0      </td><td>0          </td><td>981                </td><td>0          </td><td>981              </td><td>119130</td><td>981            </td></tr>\n",
       "<tr><td>0      </td><td>15887.0      </td><td>1 Ave & E 15 St     </td><td>47.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "<tr><td>1      </td><td>15887.0      </td><td>1 Ave & E 18 St     </td><td>40.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "<tr><td>2      </td><td>15887.0      </td><td>1 Ave & E 30 St     </td><td>42.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "<tr><td>3      </td><td>15887.0      </td><td>1 Ave & E 44 St     </td><td>12.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "<tr><td>4      </td><td>15887.0      </td><td>10 Ave & W 28 St    </td><td>28.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "<tr><td>5      </td><td>15887.0      </td><td>11 Ave & W 27 St    </td><td>30.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "<tr><td>6      </td><td>15887.0      </td><td>11 Ave & W 41 St    </td><td>20.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "<tr><td>7      </td><td>15887.0      </td><td>12 Ave & W 40 St    </td><td>13.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "<tr><td>8      </td><td>15887.0      </td><td>2 Ave & E 31 St     </td><td>33.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "<tr><td>9      </td><td>15887.0      </td><td>2 Ave & E 58 St     </td><td>54.0         </td><td>6      </td><td>Sun        </td><td>0.9354             </td><td>0.0        </td><td>22.8             </td><td>rain  </td><td>21.7           </td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th style=\"text-align: right;\">  Days</th><th>start station name  </th><th style=\"text-align: right;\">  bikes</th><th style=\"text-align: right;\">  Month</th><th>DayOfWeek  </th><th style=\"text-align: right;\">  Humidity Fraction</th><th style=\"text-align: right;\">  Rain (mm)</th><th style=\"text-align: right;\">  Temperature (C)</th><th>WC1  </th><th style=\"text-align: right;\">  Dew Point (C)</th></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>1 Ave & E 15 St     </td><td style=\"text-align: right;\">     47</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>1 Ave & E 18 St     </td><td style=\"text-align: right;\">     40</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>1 Ave & E 30 St     </td><td style=\"text-align: right;\">     42</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>1 Ave & E 44 St     </td><td style=\"text-align: right;\">     12</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>10 Ave & W 28 St    </td><td style=\"text-align: right;\">     28</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>11 Ave & W 27 St    </td><td style=\"text-align: right;\">     30</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>11 Ave & W 41 St    </td><td style=\"text-align: right;\">     20</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>12 Ave & W 40 St    </td><td style=\"text-align: right;\">     13</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>2 Ave & E 31 St     </td><td style=\"text-align: right;\">     33</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 15887</td><td>2 Ave & E 58 St     </td><td style=\"text-align: right;\">     54</td><td style=\"text-align: right;\">      6</td><td>Sun        </td><td style=\"text-align: right;\">             0.9354</td><td style=\"text-align: right;\">          0</td><td style=\"text-align: right;\">             22.8</td><td>rain </td><td style=\"text-align: right;\">           21.7</td></tr>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ----------\n",
    "# 6 - Join the weather data-per-day to the bike-starts-per-day\n",
    "print(\"Merge Daily Weather with Bikes-Per-Day\")\n",
    "bpd_with_weather = bpd.merge(wthr4,all_x=True,all_y=False)\n",
    "bpd_with_weather.describe()\n",
    "bpd_with_weather.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data has 10 columns and 83723 rows, test has 41517 rows, holdout has 14021\n",
      "\n",
      "gbm Model Build Progress: [##################################################] 100%\n",
      "\n",
      "drf Model Build Progress: [##################################################] 100%\n",
      "\n",
      "glm Model Build Progress: [##################################################] 100%\n",
      "\n",
      "deeplearning Model Build Progress: [##################################################] 100%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>Model</b></td>\n",
       "<td><b>R2 TRAIN</b></td>\n",
       "<td><b>R2 TEST</b></td>\n",
       "<td><b>R2 HOLDOUT</b></td>\n",
       "<td><b>Model Training Time (s)</b></td></tr>\n",
       "<tr><td>GBM</td>\n",
       "<td>0.9673728</td>\n",
       "<td>0.9259863</td>\n",
       "<td>0.9275258</td>\n",
       "<td>22.512</td></tr>\n",
       "<tr><td>DRF</td>\n",
       "<td>0.9004752</td>\n",
       "<td>0.8439568</td>\n",
       "<td>0.8420079</td>\n",
       "<td>46.674</td></tr>\n",
       "<tr><td>GLM</td>\n",
       "<td>0.8013288</td>\n",
       "<td>0.7958744</td>\n",
       "<td>0.8013114</td>\n",
       "<td>0.343</td></tr>\n",
       "<tr><td>DL </td>\n",
       "<td>0.9301786</td>\n",
       "<td>0.9067311</td>\n",
       "<td>0.9075087</td>\n",
       "<td>71.67</td></tr></table></div>"
      ],
      "text/plain": [
       "Model      R2 TRAIN    R2 TEST    R2 HOLDOUT    Model Training Time (s)\n",
       "-------  ----------  ---------  ------------  -------------------------\n",
       "GBM        0.967373   0.925986      0.927526                     22.512\n",
       "DRF        0.900475   0.843957      0.842008                     46.674\n",
       "GLM        0.801329   0.795874      0.801311                      0.343\n",
       "DL         0.930179   0.906731      0.907509                     71.67"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 7 - Test/Train split again, model build again, this time with weather\n",
    "split_fit_predict(bpd_with_weather)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
